﻿#region 命名空间

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Sharp.Domain.CMSBoundedContext.AritcleModule.Aggregates;
using Sharp.Domain.Repositories;
using Sharp.Infrastructure;
using Sharp.Infrastructure.Crosscutting;
using Sharp.Infrastructure.Network;

#endregion

namespace Sharp.Domain.CMSBoundedContext.AritcleModule.GatherEngine
{
    public class GatherEngineMain : IGatherEngineMain
    {
        private readonly IRepositoryContext _repositoryContext;
        private static IGatherRepository _gatherRepository;
        private static IGatherUrlCacheRepository _gatherUrlCacheRepository;
        public GatherEngineMain(IRepositoryContext repositoryContext, IGatherRepository gatherRepository, IGatherUrlCacheRepository gatherUrlCacheRepository)
        {
            _repositoryContext = repositoryContext;
            _gatherRepository = gatherRepository;
            _gatherUrlCacheRepository = gatherUrlCacheRepository;

            ContentPageRules = new List<Regex>();
        }

        /// <summary>
        /// 采集栏目页面
        /// </summary>
        /// <param name="cp"></param>
        /// <returns></returns>
        public ClassGatherResult GatherCategoryPage(CategoryParameter cp)
        {
            if (cp.NextCategoryUrl == null)
            {
                cp.NextCategoryUrl = string.Empty;
            }
            var rulesList = _gatherRepository.GetRules(cp.IDs);
            if (!string.IsNullOrEmpty(rulesList.First().ContentPageRule))
            {

                ContentPageRules = rulesList.First().ContentPageRule.Split(new[] { '\r', '\n' },
                       StringSplitOptions.RemoveEmptyEntries).Select(x => new Regex(x, RegexOptions.IgnoreCase))
                           .ToList();
            }

            List<ArrStartUrls> startUrls = new List<ArrStartUrls>();


            var q = from x in rulesList where x.ParentId == null select new ArrStartUrls { GatherId = x.Id, Url = x.StartUrl };

            foreach (var v in q)
            {
                var vv = from x in v.Url.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries)
                         select new ArrStartUrls { GatherId = v.GatherId, Url = x };

                startUrls.AddRange(vv);
            }


            if (cp.StartUrlIndex >= startUrls.Count)
            {
                cp.StartUrlIndex = startUrls.Count - 1;
            }

            string startUrl = !cp.NextCategoryUrl.IsNullOrEmpty() ? cp.NextCategoryUrl : startUrls[cp.StartUrlIndex].Url;
            ClassGatherResult cgr = new ClassGatherResult
            {
                GatherId = startUrls[cp.StartUrlIndex].GatherId,
                GatheringUrl = startUrl
            };

            try
            {
                string[] getPageListRs = new string[0];
                HtmlDocument doc;
                if (cp.IsReverse == false)
                {
                    getPageListRs = GetArticleUrls(startUrls[cp.StartUrlIndex].GatherId, startUrl, rulesList, out doc);
                    if (doc == null)
                    {
                        cgr.Errors = "采集文章列表，出现网络错误。";
                        goto Exit;
                    }
                    HtmlNode htmlnode = doc.DocumentNode;

                    #region 得到下一页链接

                    foreach (
                        var ag in
                            rulesList.Where(
                                m =>
                                    cp.IsReverse == false && !string.IsNullOrEmpty(m.CategoryPageNextUrl) &&
                                    (m.Id == startUrls[cp.StartUrlIndex].GatherId ||
                                     m.ParentId == startUrls[cp.StartUrlIndex].GatherId)))
                    {
                        switch (GatherRule.Analysis(ag.CategoryPageNextUrl))
                        {
                            case GatherRuleType.XPath:
                                HtmlNode hn = htmlnode.SelectSingleNode(ag.CategoryPageNextUrl.Substring(2));
                                cgr.NextCategoryUrl = hn == null ? string.Empty : hn.Attributes["href"].Value;
                                break;
                            case GatherRuleType.DomQuery:
                                hn = doc.SingleGet(ag.CategoryPageNextUrl.Substring(2));
                                if (hn == null || hn.Attributes["href"] == null)
                                {
                                    cgr.NextCategoryUrl = string.Empty;
                                }
                                else
                                {
                                    cgr.NextCategoryUrl = hn.Attributes["href"].Value;
                                }
                                break;
                            default:
                                cgr.NextCategoryUrl =
                                    Utilities.GetInfos(htmlnode.OuterHtml, ag.CategoryPageNextUrl)[0];
                                break;
                        }
                        if (!cgr.NextCategoryUrl.IsNullOrEmpty()) //格式化采集链接
                        {
                            cgr.NextCategoryUrl = Utilities.FormatUrl(startUrl, cgr.NextCategoryUrl);
                        }

                        if (cgr.NextCategoryUrl.Equals(startUrl, StringComparison.CurrentCultureIgnoreCase))
                            //防止最后一页的“下一页“就是本身
                        {
                            cgr.NextCategoryUrl = ""; //当作没采集到
                        }

                        if (!cgr.NextCategoryUrl.IsNullOrEmpty())
                        {
                            break; //找到下一页，跳出循环。
                        }
                    }

                    #endregion

                }
                else
                {
                    string[] classNextPages = _gatherUrlCacheRepository.ReadNextPages(startUrl);
                    if (classNextPages.Length == 0)
                    {
                        classNextPages = GetNPageUrl(rulesList, startUrls[cp.StartUrlIndex].GatherId, startUrl);
                        _gatherUrlCacheRepository.CacheNextPages(startUrl, classNextPages);
                    }

                    foreach (var url in classNextPages)
                    {
                        string[] arr = GetArticleUrls(startUrls[cp.StartUrlIndex].GatherId, url, rulesList, out doc);
                        getPageListRs = getPageListRs.Union(arr).ToArray();
                    }
                }

                #region 是否换下一个链接采集

                //采集不到“下一页”地址就跳到下一链接进行采集
                if ((cp.StartUrlIndex < startUrls.Count - 1) && string.IsNullOrEmpty(cgr.NextCategoryUrl))
                {
                    cgr.GatherId = startUrls[cp.StartUrlIndex].GatherId;
                    cp.StartUrlIndex++;
                    cgr.NextCategoryUrl = startUrls[cp.StartUrlIndex].Url; //换下一个链接采集
                }
                if ((cp.StartUrlIndex == startUrls.Count - 1) && string.IsNullOrEmpty(cgr.NextCategoryUrl))
                {
                    cgr.GatherId = startUrls[cp.StartUrlIndex].GatherId;
                    cp.StartUrlIndex++;
                    cgr.NextCategoryUrl = string.Empty;
                }

                #endregion

                Exit:
                cgr.StartUrlIndex = cp.StartUrlIndex;
                cgr.GatherNum = 0;
                if (string.IsNullOrWhiteSpace(cgr.Errors))
                    cgr.Errors = string.Empty;

                getPageListRs = getPageListRs.Distinct().ToArray();

                if (cp.IsReverse)
                {
                    getPageListRs = getPageListRs.Reverse().ToArray();
                }
                var list = getPageListRs.Select(x => Utilities.FormatUrl(startUrl, x)).ToList();
                if (cgr.Urls == null)
                {
                    cgr.Urls = new List<string>();
                }
                list.ForEach(url =>
                {
                    if (!ContentPageRules.Any())
                    {
                        cgr.Urls.Add(url);
                    }
                    else
                    {
                        if (ContentPageRules.Any(reg => reg.IsMatch(url)))
                        {
                            cgr.Urls.Add(url);
                        }
                    }
                });

                return cgr;
            }
            catch (Exception ex)
            {
                LogHelper.Error(ex);
                cgr.Errors = ex.Message;
                return cgr;
            }
        }

        #region 收集“下一页”

        /// <summary>
        /// 收集“下一页”
        /// </summary>
        /// <param name="rulesList"></param>
        /// <param name="gatherId"></param>
        /// <param name="startUrl"></param>
        /// <returns>返回栏目后5页</returns>
        private string[] GetNPageUrl(IList<Gather> rulesList, Guid gatherId, string startUrl)
        {
            NetHelper engine = new NetHelper
            {
                EncodingString = rulesList.First(x => x.Id == gatherId).Encoding
            };
            IList<string> classUrls = new List<string>();

            string nextCategoryUrl = startUrl;

            int count = 1;
            do
            {
                try
                {
                    if (count > 5)
                    {
                        break;
                    }
                    RemoteRes info = engine.Get(nextCategoryUrl);
                    if (info.StatusCode != HttpStatusCode.OK)
                    {
                        return classUrls.ToArray();
                    }
                    classUrls.Insert(0, nextCategoryUrl);
                    HtmlDocument doc = new HtmlDocument();
                    //载入HTML
                    doc.LoadHtml(info.Html);

                    HtmlNode htmlnode = doc.DocumentNode;

                    #region 通过采集文章页确认该 url 是否有意义。

                    string[] getPageListRs = new string[0];
                    foreach (
                        var ag in
                            rulesList.Where(m => !m.ContentPageUrl.IsNullOrEmpty() && (m.Id == gatherId || m.ParentId == gatherId))
                        )
                    {
                        IList<HtmlNode> hc;
                        if (GatherRule.Analysis(ag.ContentPageUrl) == GatherRuleType.XPath)
                        {
                            hc = htmlnode.SelectNodes(ag.ContentPageUrl.Substring(2));

                            //得到文章页链接数组,它可能是相对链接
                            getPageListRs = hc.Attr("href");
                        }
                        else if (GatherRule.Analysis(ag.ContentPageUrl) == GatherRuleType.DomQuery)
                        {
                            hc = doc.Get(ag.ContentPageUrl.Substring(2));

                            //得到文章页链接数组,它可能是相对链接
                            getPageListRs = hc.Attr("href");
                        }
                        else
                        {
                            //得到栏目页扫描范围
                            string strCategoryArea = "";
                            foreach (
                                var v in
                                    rulesList.Where(
                                        m => !m.CategoryArea.IsNullOrEmpty() && (m.Id == gatherId || m.ParentId == gatherId)))
                            {
                                strCategoryArea = Utilities.GetInfo(info.Html, v.CategoryArea);
                                if (string.IsNullOrWhiteSpace(strCategoryArea) == false)
                                {
                                    break;
                                }
                            }

                            getPageListRs = Utilities.GetInfos(strCategoryArea, ag.ContentPageUrl);
                        }
                        if (getPageListRs.Length > 0)
                            break;
                    }

                    if (getPageListRs.Length <= 0)
                    {
                        break;
                    }

                    #endregion

                    foreach (
                        var ag in
                            rulesList.Where(
                                m => !m.CategoryPageNextUrl.IsNullOrEmpty() && (m.Id == gatherId || m.ParentId == gatherId)))
                    {
                        switch (GatherRule.Analysis(ag.CategoryPageNextUrl))
                        {
                            case GatherRuleType.XPath:
                                HtmlNode hn = htmlnode.SelectSingleNode(ag.CategoryPageNextUrl.Substring(2));
                                nextCategoryUrl = hn == null ? string.Empty : hn.Attributes["href"].Value;
                                break;
                            case GatherRuleType.DomQuery:
                                hn = doc.SingleGet(ag.CategoryPageNextUrl.Substring(2));
                                nextCategoryUrl = hn == null ? string.Empty : hn.Attributes["href"].Value;
                                break;
                            default:
                                nextCategoryUrl = Utilities.GetInfos(info.Html, ag.CategoryPageNextUrl)[0];
                                break;
                        }
                        if (!nextCategoryUrl.IsNullOrEmpty()) //格式化采集链接
                        {
                            break;
                        }
                    }
                    if (!nextCategoryUrl.IsNullOrEmpty()) //格式化采集链接
                    {
                        nextCategoryUrl = Utilities.FormatUrl(startUrl, nextCategoryUrl);
                        count++;
                    }
                    else
                    {
                        break;
                    }
                    if (nextCategoryUrl.EndsWith("#") || startUrl == nextCategoryUrl)
                    {
                        break;
                    }
                }
                catch
                {
                    count++;
                }
            } while (!nextCategoryUrl.IsNullOrEmpty());

            return classUrls.ToArray();
        }

        #endregion

        #region 获取栏目页的所有文章地址

        /// <summary>
        /// 获取栏目页的所有文章地址
        /// </summary>
        /// <param name="gatherId"></param>
        /// <param name="startUrl"></param>
        /// <param name="rulesList"></param>
        /// <param name="doc"></param>
        /// <returns></returns>
        private string[] GetArticleUrls(Guid gatherId, string startUrl, IList<Gather> rulesList, out HtmlDocument doc)
        {
            NetHelper engine = new NetHelper
            {
                EncodingString = rulesList.First(x => x.Id == gatherId).Encoding
            };
            //得到栏目网页代码
            RemoteRes info = engine.Get(startUrl);
            if (info.StatusCode != HttpStatusCode.OK)
            {
                doc = null;
                return new string[] { };
            }
            doc = new HtmlDocument();
            //载入HTML
            doc.LoadHtml(info.Html);

            HtmlNode htmlnode = doc.DocumentNode;
            string[] getPageListRs = new string[0];
            foreach (
                var ag in rulesList.Where(m => !m.ContentPageUrl.IsNullOrEmpty() && (m.Id == gatherId || m.ParentId == gatherId)))
            {
                IList<HtmlNode> hc;
                if (GatherRule.Analysis(ag.ContentPageUrl) == GatherRuleType.XPath)
                {
                    hc = htmlnode.SelectNodes(ag.ContentPageUrl.Substring(2));

                    //得到文章页链接数组,它可能是相对链接
                    getPageListRs = hc.Attr("href");
                }
                else if (GatherRule.Analysis(ag.ContentPageUrl) == GatherRuleType.DomQuery)
                {
                    hc = doc.Get(ag.ContentPageUrl.Substring(2));

                    //得到文章页链接数组,它可能是相对链接
                    getPageListRs = hc.Attr("href");
                }
                else
                {
                    //得到栏目页扫描范围
                    string strCategoryArea = "";
                    foreach (
                        var v in
                            rulesList.Where(m => m.CategoryArea.Length > 2 && (m.Id == gatherId || m.ParentId == gatherId)))
                    {
                        strCategoryArea = Utilities.GetInfo(info.Html, v.CategoryArea);
                        if (string.IsNullOrWhiteSpace(strCategoryArea) == false)
                        {
                            break;
                        }
                    }

                    //得到文章页链接数组,它可能是相对链接
                    getPageListRs = Utilities.GetInfos(strCategoryArea, ag.ContentPageUrl);
                }
                if (getPageListRs.Length > 0)
                    break;
            }

            return
                getPageListRs.Where(x => !x.StartsWith("javascript", StringComparison.CurrentCultureIgnoreCase))
                             .ToArray();
        }

        #endregion

        /// <summary>
        /// 内容页路径规则
        /// </summary>
        private List<Regex> ContentPageRules { get; set; }
    }

    public class ArrStartUrls
    {
        public string Url { get; set; }
        public Guid GatherId { get; set; }
    }
}